{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Summarizing_Text_with_T5.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "311dd5614ff1456fa608b06c6a486333": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_483e8e39248842cca6fdb69497417033",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_898bb243539e4693a1a024e54dac42e2",
              "IPY_MODEL_ab283be8197e4a638fd83dc475b57486"
            ]
          }
        },
        "483e8e39248842cca6fdb69497417033": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "898bb243539e4693a1a024e54dac42e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_af7cb58500024f7cac6545a44045bdd1",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1200,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1200,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d252ead0d3fc4885bed21c592dc9394c"
          }
        },
        "ab283be8197e4a638fd83dc475b57486": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8ed15a3ae7c64fdea663832082e80529",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.20k/1.20k [03:27&lt;00:00, 5.78B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6a4014978e96442789889fef7919d87f"
          }
        },
        "af7cb58500024f7cac6545a44045bdd1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d252ead0d3fc4885bed21c592dc9394c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8ed15a3ae7c64fdea663832082e80529": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6a4014978e96442789889fef7919d87f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "173d4c16ef284a6085b183da2be8574e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_15d71a8c88ab4dc2bfb20853b5bac3d3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fef9e2aa436040c0822605be0df0c359",
              "IPY_MODEL_9b0393c7c87e42838da0e38fb72b2941"
            ]
          }
        },
        "15d71a8c88ab4dc2bfb20853b5bac3d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fef9e2aa436040c0822605be0df0c359": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3707cb9b877c451bb245d051c19af8fa",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2950825948,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2950825948,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_28ac8f3a26ca4c51ab913c482234f3ab"
          }
        },
        "9b0393c7c87e42838da0e38fb72b2941": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3ef057edd14f4fafbbb0b6fb81e9ba97",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2.95G/2.95G [03:27&lt;00:00, 14.2MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8ff347600d6f42e5bf198232909fe086"
          }
        },
        "3707cb9b877c451bb245d051c19af8fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "28ac8f3a26ca4c51ab913c482234f3ab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3ef057edd14f4fafbbb0b6fb81e9ba97": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8ff347600d6f42e5bf198232909fe086": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "31a90525042744a49f8d48305354b9ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_55811622761d49559291f3e74f072d31",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3c68af00a0784efca7a3dce29c060eef",
              "IPY_MODEL_07acd3ae43eb4dfb893cc65e72320367"
            ]
          }
        },
        "55811622761d49559291f3e74f072d31": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3c68af00a0784efca7a3dce29c060eef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4a1b68c322254a53a77e4d05e9ee7631",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 791656,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791656,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e7c516f923f74666a8a4b11e17c333bd"
          }
        },
        "07acd3ae43eb4dfb893cc65e72320367": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0be43a9c00324bb9abeedeb2a67b883a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 792k/792k [00:00&lt;00:00, 1.68MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_369a4af884f3436683d68ec126d7b6ec"
          }
        },
        "4a1b68c322254a53a77e4d05e9ee7631": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e7c516f923f74666a8a4b11e17c333bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0be43a9c00324bb9abeedeb2a67b883a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "369a4af884f3436683d68ec126d7b6ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RcdcqBkV0MTU"
      },
      "source": [
        "#Summarizing Text with T5\n",
        "Copyright 2020, Denis Rothman. MIT License. Hugging Face usage example was modified for educational purposes.\n",
        "\n",
        "[Hugging Face Models](https://huggingface.co/transformers/model_doc/t5.html)\n",
        "\n",
        "[Hugging Face Framework Usage](https://huggingface.co/transformers/usage.html)\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "06QFZGxsf_KJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4bb34ef6-44f5-4ae5-d334-8fa716bf9656"
      },
      "source": [
        "!pip install transformers==4.0.0"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers==4.0.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/99/84/7bc03215279f603125d844bf81c3fb3f2d50fe8e511546eb4897e4be2067/transformers-4.0.0-py3-none-any.whl (1.4MB)\n",
            "\u001b[K     |████████████████████████████████| 1.4MB 9.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==4.0.0) (3.0.12)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==4.0.0) (2019.12.20)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers==4.0.0) (0.8)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers==4.0.0) (4.41.1)\n",
            "Collecting tokenizers==0.9.4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0f/1c/e789a8b12e28be5bc1ce2156cf87cb522b379be9cadc7ad8091a4cc107c4/tokenizers-0.9.4-cp36-cp36m-manylinux2010_x86_64.whl (2.9MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9MB 26.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers==4.0.0) (2.23.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers==4.0.0) (20.4)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 41.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers==4.0.0) (1.18.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.0.0) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.0.0) (2020.11.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.0.0) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.0.0) (1.24.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->transformers==4.0.0) (1.15.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers==4.0.0) (2.4.7)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==4.0.0) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==4.0.0) (0.17.0)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893257 sha256=cd41d8610aac8d370f5d6a1c57d73172911ecb039b55e1a3e3f8780a2420a753\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.9.4 transformers-4.0.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3tYFf-cEIkKL",
        "outputId": "6061799b-9e78-42c5-9902-d268217273e4"
      },
      "source": [
        "!pip install sentencepiece==0.1.94"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting sentencepiece==0.1.94\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e5/2d/6d4ca4bef9a67070fa1cac508606328329152b1df10bdf31fb6e4e727894/sentencepiece-0.1.94-cp36-cp36m-manylinux2014_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 9.5MB/s \n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.94\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FEQO4tDl7xH_"
      },
      "source": [
        "display_architecture=True"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q8suV48O07TW",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235,
          "referenced_widgets": [
            "311dd5614ff1456fa608b06c6a486333",
            "483e8e39248842cca6fdb69497417033",
            "898bb243539e4693a1a024e54dac42e2",
            "ab283be8197e4a638fd83dc475b57486",
            "af7cb58500024f7cac6545a44045bdd1",
            "d252ead0d3fc4885bed21c592dc9394c",
            "8ed15a3ae7c64fdea663832082e80529",
            "6a4014978e96442789889fef7919d87f",
            "173d4c16ef284a6085b183da2be8574e",
            "15d71a8c88ab4dc2bfb20853b5bac3d3",
            "fef9e2aa436040c0822605be0df0c359",
            "9b0393c7c87e42838da0e38fb72b2941",
            "3707cb9b877c451bb245d051c19af8fa",
            "28ac8f3a26ca4c51ab913c482234f3ab",
            "3ef057edd14f4fafbbb0b6fb81e9ba97",
            "8ff347600d6f42e5bf198232909fe086",
            "31a90525042744a49f8d48305354b9ec",
            "55811622761d49559291f3e74f072d31",
            "3c68af00a0784efca7a3dce29c060eef",
            "07acd3ae43eb4dfb893cc65e72320367",
            "4a1b68c322254a53a77e4d05e9ee7631",
            "e7c516f923f74666a8a4b11e17c333bd",
            "0be43a9c00324bb9abeedeb2a67b883a",
            "369a4af884f3436683d68ec126d7b6ec"
          ]
        },
        "outputId": "cb65b089-f9f1-423c-8c35-f5f39bc7c139"
      },
      "source": [
        "import torch\n",
        "import json \n",
        "from transformers import T5Tokenizer, T5ForConditionalGeneration, T5Config\n",
        "\n",
        "model = T5ForConditionalGeneration.from_pretrained('t5-large')\n",
        "tokenizer = T5Tokenizer.from_pretrained('t5-large')\n",
        "device = torch.device('cpu')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "311dd5614ff1456fa608b06c6a486333",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1200.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "173d4c16ef284a6085b183da2be8574e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=2950825948.0, style=ProgressStyle(descr…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at t5-large were not used when initializing T5ForConditionalGeneration: ['decoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weight']\n",
            "- This IS expected if you are initializing T5ForConditionalGeneration from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing T5ForConditionalGeneration from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "31a90525042744a49f8d48305354b9ec",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=791656.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q6zHDK7I1GsY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "21dc9317-47da-414e-b6fb-a24f60e1cda6"
      },
      "source": [
        "if display_architecture==True:\n",
        " print(model.config)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "T5Config {\n",
            "  \"_name_or_path\": \"t5-large\",\n",
            "  \"architectures\": [\n",
            "    \"T5WithLMHeadModel\"\n",
            "  ],\n",
            "  \"d_ff\": 4096,\n",
            "  \"d_kv\": 64,\n",
            "  \"d_model\": 1024,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"dropout_rate\": 0.1,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"feed_forward_proj\": \"relu\",\n",
            "  \"initializer_factor\": 1.0,\n",
            "  \"is_encoder_decoder\": true,\n",
            "  \"layer_norm_epsilon\": 1e-06,\n",
            "  \"model_type\": \"t5\",\n",
            "  \"n_positions\": 512,\n",
            "  \"num_decoder_layers\": 24,\n",
            "  \"num_heads\": 16,\n",
            "  \"num_layers\": 24,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"relative_attention_num_buckets\": 32,\n",
            "  \"task_specific_params\": {\n",
            "    \"summarization\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"length_penalty\": 2.0,\n",
            "      \"max_length\": 200,\n",
            "      \"min_length\": 30,\n",
            "      \"no_repeat_ngram_size\": 3,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"summarize: \"\n",
            "    },\n",
            "    \"translation_en_to_de\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to German: \"\n",
            "    },\n",
            "    \"translation_en_to_fr\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to French: \"\n",
            "    },\n",
            "    \"translation_en_to_ro\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to Romanian: \"\n",
            "    }\n",
            "  },\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 32128\n",
            "}\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5LaWN15NPIPC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "623480ce-b408-462b-a8c5-5c12959f724c"
      },
      "source": [
        "if(display_architecture==True):\n",
        "  print(model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "T5ForConditionalGeneration(\n",
            "  (shared): Embedding(32128, 1024)\n",
            "  (encoder): T5Stack(\n",
            "    (embed_tokens): Embedding(32128, 1024)\n",
            "    (block): ModuleList(\n",
            "      (0): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (relative_attention_bias): Embedding(32, 16)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (1): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (2): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (3): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (4): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (5): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (6): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (7): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (8): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (9): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (10): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (11): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (12): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (13): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (14): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (15): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (16): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (17): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (18): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (19): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (20): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (21): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (22): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (23): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (final_layer_norm): T5LayerNorm()\n",
            "    (dropout): Dropout(p=0.1, inplace=False)\n",
            "  )\n",
            "  (decoder): T5Stack(\n",
            "    (embed_tokens): Embedding(32128, 1024)\n",
            "    (block): ModuleList(\n",
            "      (0): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (relative_attention_bias): Embedding(32, 16)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (1): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (2): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (3): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (4): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (5): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (6): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (7): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (8): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (9): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (10): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (11): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (12): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (13): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (14): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (15): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (16): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (17): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (18): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (19): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (20): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (21): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (22): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (23): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (final_layer_norm): T5LayerNorm()\n",
            "    (dropout): Dropout(p=0.1, inplace=False)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1024, out_features=32128, bias=False)\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DS2twf1P1UYI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e060a702-cf27-41b4-d162-ffb21872b81c"
      },
      "source": [
        "if display_architecture==True:\n",
        "  print(model.encoder)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "T5Stack(\n",
            "  (embed_tokens): Embedding(32128, 1024)\n",
            "  (block): ModuleList(\n",
            "    (0): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (relative_attention_bias): Embedding(32, 16)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (1): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (2): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (3): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (4): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (5): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (6): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (7): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (8): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (9): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (10): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (11): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (12): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (13): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (14): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (15): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (16): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (17): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (18): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (19): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (20): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (21): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (22): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (23): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (final_layer_norm): T5LayerNorm()\n",
            "  (dropout): Dropout(p=0.1, inplace=False)\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MCwdhX9U1MA5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ae39b023-77ec-483d-fb33-2bc59d2c0996"
      },
      "source": [
        "if display_architecture==True:\n",
        "  print(model.decoder)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "T5Stack(\n",
            "  (embed_tokens): Embedding(32128, 1024)\n",
            "  (block): ModuleList(\n",
            "    (0): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (relative_attention_bias): Embedding(32, 16)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (1): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (2): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (3): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (4): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (5): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (6): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (7): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (8): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (9): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (10): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (11): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (12): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (13): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (14): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (15): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (16): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (17): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (18): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (19): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (20): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (21): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (22): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (23): T5Block(\n",
            "      (layer): ModuleList(\n",
            "        (0): T5LayerSelfAttention(\n",
            "          (SelfAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (1): T5LayerCrossAttention(\n",
            "          (EncDecAttention): T5Attention(\n",
            "            (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (2): T5LayerFF(\n",
            "          (DenseReluDense): T5DenseReluDense(\n",
            "            (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "            (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (layer_norm): T5LayerNorm()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (final_layer_norm): T5LayerNorm()\n",
            "  (dropout): Dropout(p=0.1, inplace=False)\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GmrCDtcL1hPn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d93ba363-7336-4514-b693-b4c33dd8cb07"
      },
      "source": [
        "if display_architecture==True:\n",
        "  print(model.forward)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<bound method T5ForConditionalGeneration.forward of T5ForConditionalGeneration(\n",
            "  (shared): Embedding(32128, 1024)\n",
            "  (encoder): T5Stack(\n",
            "    (embed_tokens): Embedding(32128, 1024)\n",
            "    (block): ModuleList(\n",
            "      (0): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (relative_attention_bias): Embedding(32, 16)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (1): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (2): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (3): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (4): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (5): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (6): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (7): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (8): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (9): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (10): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (11): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (12): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (13): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (14): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (15): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (16): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (17): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (18): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (19): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (20): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (21): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (22): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (23): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (final_layer_norm): T5LayerNorm()\n",
            "    (dropout): Dropout(p=0.1, inplace=False)\n",
            "  )\n",
            "  (decoder): T5Stack(\n",
            "    (embed_tokens): Embedding(32128, 1024)\n",
            "    (block): ModuleList(\n",
            "      (0): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (relative_attention_bias): Embedding(32, 16)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (1): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (2): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (3): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (4): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (5): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (6): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (7): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (8): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (9): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (10): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (11): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (12): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (13): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (14): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (15): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (16): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (17): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (18): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (19): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (20): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (21): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (22): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (23): T5Block(\n",
            "        (layer): ModuleList(\n",
            "          (0): T5LayerSelfAttention(\n",
            "            (SelfAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): T5LayerCrossAttention(\n",
            "            (EncDecAttention): T5Attention(\n",
            "              (q): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (k): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (v): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "              (o): Linear(in_features=1024, out_features=1024, bias=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): T5LayerFF(\n",
            "            (DenseReluDense): T5DenseReluDense(\n",
            "              (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",
            "              (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (layer_norm): T5LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (final_layer_norm): T5LayerNorm()\n",
            "    (dropout): Dropout(p=0.1, inplace=False)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=1024, out_features=32128, bias=False)\n",
            ")>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S5KfhCrifP01"
      },
      "source": [
        "\n",
        "def summarize(text,ml):\n",
        "  preprocess_text = text.strip().replace(\"\\n\",\"\")\n",
        "  t5_prepared_Text = \"summarize: \"+preprocess_text\n",
        "  print (\"Preprocessed and prepared text: \\n\", t5_prepared_Text)\n",
        "\n",
        "  tokenized_text = tokenizer.encode(t5_prepared_Text, return_tensors=\"pt\").to(device)\n",
        "\n",
        "  # summmarize \n",
        "  summary_ids = model.generate(tokenized_text,\n",
        "                                      num_beams=4,\n",
        "                                      no_repeat_ngram_size=2,\n",
        "                                      min_length=30,\n",
        "                                      max_length=ml,\n",
        "                                      early_stopping=True)\n",
        "\n",
        "  output = tokenizer.decode(summary_ids[0], skip_special_tokens=True)\n",
        "  return output"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vqiTNoDc7pOv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d46bc587-b39d-4e8e-bd5e-3d4d6d529168"
      },
      "source": [
        "text=\"\"\"\n",
        "The United States Declaration of Independence was the first Etext\n",
        "released by Project Gutenberg, early in 1971.  The title was stored\n",
        "in an emailed instruction set which required a tape or diskpack be\n",
        "hand mounted for retrieval.  The diskpack was the size of a large\n",
        "cake in a cake carrier, cost $1500, and contained 5 megabytes, of\n",
        "which this file took 1-2%.  Two tape backups were kept plus one on\n",
        "paper tape.  The 10,000 files we hope to have online by the end of\n",
        "2001 should take about 1-2% of a comparably priced drive in 2001.\n",
        "\"\"\"\n",
        "print(\"Number of characters:\",len(text))\n",
        "summary=summarize(text,50)\n",
        "print (\"\\n\\nSummarized text: \\n\",summary)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of characters: 534\n",
            "Preprocessed and prepared text: \n",
            " summarize: The United States Declaration of Independence was the first Etextreleased by Project Gutenberg, early in 1971.  The title was storedin an emailed instruction set which required a tape or diskpack behand mounted for retrieval.  The diskpack was the size of a largecake in a cake carrier, cost $1500, and contained 5 megabytes, ofwhich this file took 1-2%.  Two tape backups were kept plus one onpaper tape.  The 10,000 files we hope to have online by the end of2001 should take about 1-2% of a comparably priced drive in 2001.\n",
            "\n",
            "\n",
            "Summarized text: \n",
            " the united states declaration of independence was the first etext published by project gutenberg, early in 1971. the 10,000 files we hope to have online by the end of2001 should take about 1-2% of a comparably priced drive in\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2321zS1Q3jPX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fccf0c6e-926f-4f65-b3ba-b0e19715f87b"
      },
      "source": [
        "#Bill of Rights,V\n",
        "text =\"\"\"\n",
        "No person shall be held to answer for a capital, or otherwise infamous crime,\n",
        "unless on a presentment or indictment of a Grand Jury, except in cases arising\n",
        "in the land or naval forces, or in the Militia, when in actual service\n",
        "in time of War or public danger; nor shall any person be subject for\n",
        "the same offense to be twice put in jeopardy of life or limb;\n",
        "nor shall be compelled in any criminal case to be a witness against himself,\n",
        "nor be deprived of life, liberty, or property, without due process of law;\n",
        "nor shall private property be taken for public use without just compensation.\n",
        "\n",
        "\"\"\"\n",
        "print(\"Number of characters:\",len(text))\n",
        "summary=summarize(text,50)\n",
        "print (\"\\n\\nSummarized text: \\n\",summary)\n",
        " "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of characters: 591\n",
            "Preprocessed and prepared text: \n",
            " summarize: No person shall be held to answer for a capital, or otherwise infamous crime,unless on a presentment or indictment of a Grand Jury, except in cases arisingin the land or naval forces, or in the Militia, when in actual servicein time of War or public danger; nor shall any person be subject forthe same offense to be twice put in jeopardy of life or limb;nor shall be compelled in any criminal case to be a witness against himself,nor be deprived of life, liberty, or property, without due process of law;nor shall private property be taken for public use without just compensation.\n",
            "\n",
            "\n",
            "Summarized text: \n",
            " no person shall be held to answer for a capital, or otherwise infamous crime, unless ona presentment or indictment ofa Grand Jury. nor shall any person be subject for the same offense to be twice put\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k_h8oQ55_zr5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7c957d5a-6711-4169-9244-51562a6cc9cd"
      },
      "source": [
        "#Montana Corporate Law\n",
        "#https://corporations.uslegal.com/state-corporation-law/montana-corporation-law/#:~:text=Montana%20Corporation%20Law,carrying%20out%20its%20business%20activities.\n",
        "\n",
        "text =\"\"\"The law regarding corporations prescribes that a corporation can be incorporated in the state of Montana to serve any lawful purpose.  In the state of Montana, a corporation has all the powers of a natural person for carrying out its business activities.  The corporation can sue and be sued in its corporate name.  It has perpetual succession.  The corporation can buy, sell or otherwise acquire an interest in a real or personal property.  It can conduct business, carry on operations, and have offices and exercise the powers in a state, territory or district in possession of the U.S., or in a foreign country.  It can appoint officers and agents of the corporation for various duties and fix their compensation.\n",
        "The name of a corporation must contain the word “corporation” or its abbreviation “corp.”  The name of a corporation should not be deceptively similar to the name of another corporation incorporated in the same state.  It should not be deceptively identical to the fictitious name adopted by a foreign corporation having business transactions in the state.\n",
        "The corporation is formed by one or more natural persons by executing and filing articles of incorporation to the secretary of state of filing.  The qualifications for directors are fixed either by articles of incorporation or bylaws.  The names and addresses of the initial directors and purpose of incorporation should be set forth in the articles of incorporation.  The articles of incorporation should contain the corporate name, the number of shares authorized to issue, a brief statement of the character of business carried out by the corporation, the names and addresses of the directors until successors are elected, and name and addresses of incorporators.  The shareholders have the power to change the size of board of directors.\n",
        "\"\"\"\n",
        "print(\"Number of characters:\",len(text))\n",
        "summary=summarize(text,50)\n",
        "print (\"\\n\\nSummarized text: \\n\",summary)\n",
        " "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of characters: 1816\n",
            "Preprocessed and prepared text: \n",
            " summarize: The law regarding corporations prescribes that a corporation can be incorporated in the state of Montana to serve any lawful purpose.  In the state of Montana, a corporation has all the powers of a natural person for carrying out its business activities.  The corporation can sue and be sued in its corporate name.  It has perpetual succession.  The corporation can buy, sell or otherwise acquire an interest in a real or personal property.  It can conduct business, carry on operations, and have offices and exercise the powers in a state, territory or district in possession of the U.S., or in a foreign country.  It can appoint officers and agents of the corporation for various duties and fix their compensation.The name of a corporation must contain the word “corporation” or its abbreviation “corp.”  The name of a corporation should not be deceptively similar to the name of another corporation incorporated in the same state.  It should not be deceptively identical to the fictitious name adopted by a foreign corporation having business transactions in the state.The corporation is formed by one or more natural persons by executing and filing articles of incorporation to the secretary of state of filing.  The qualifications for directors are fixed either by articles of incorporation or bylaws.  The names and addresses of the initial directors and purpose of incorporation should be set forth in the articles of incorporation.  The articles of incorporation should contain the corporate name, the number of shares authorized to issue, a brief statement of the character of business carried out by the corporation, the names and addresses of the directors until successors are elected, and name and addresses of incorporators.  The shareholders have the power to change the size of board of directors.\n",
            "\n",
            "\n",
            "Summarized text: \n",
            " a corporation can be incorporated in the state of Montana to serve any lawful purpose. the corporation has perpetual succession and can sue and be sued in its corporate name. it can conduct business, carry on operations, and have offices\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}
